#! /usr/bin/env python
#coding=utf-8

"""

  2011 Boris Tatarintsev

  Contains API for
  1. Easily login to any LJ account
  2. Get friends list for user
  3. Get user posts

"""

import urllib
import urllib2
import hashlib
import re
import copy
import time
import thread
import threading

from funclib import *
from hashtree import *

# exceptions types

class ServerError(BaseException):
	pass

class AuthError(BaseException):
	pass

class ParseError(BaseException):
	pass

class SocketError(BaseException):
    pass

class MyThread (threading.Thread):
    
    def __init__(self, threadID, name, counter):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.counter = counter
    
    def setParams(self, f, *args):        
        self.args = args[:]
        self.f = f        
    
    def run(self):
        #print "Starting " + self.name
        #print self.args[0], self.args[1]
        self.args[1][self.args[0]] = self.f(self.args[0], self.args[2], self.args[3])
        #print "Exiting " + self.name        

    
class LJUserSession:    

    threadLock = threading.Lock()
    
    # regexp to extract friends references HTML code    
    reFriendsList = re.compile(".*\<table.*?\>\<p\>(.*)\<\/p\>\<\/table\>.*")
    reFetchFriends = re.compile("\<a href\s*?\=\s*?\'(http\:\/\/.*?)\'.*?\>(.*?)\<\/a\>")
    reCheckCommunity = re.compile(".*?community\.livejournal\.com.*?")        
    
    def __init__(self, username, password):
        self.flatServerUrl = u"http://livejournal.com/interface/flat"
        self.authServerUrl = u"http://www.livejournal.com/login.bml"
        self.profileUrl = u"http://www.livejournal.com/profile.bml"
        
        self.m_userName = username
        self.m_password = password
        
        self.m_verbose = True
        
        # threads params
        self.m_maxThreads = 1        
            
    def __getChalResponse(self):
        # get authorization response from the server
        dataDict = {"mode": "getchallenge"}
        data = urllib.urlencode(dataDict)
        request = urllib2.Request(self.flatServerUrl, data)
        
        try:
            fp = urllib2.urlopen(request)
        except:
            raise SocketError
            
        text = fp.read()        
        fp.close()        
        return self.__parseAuthResponse(text)

    def __parseAuthResponse(self, textResponse):
        # parses authorization response into hash
        elements = textResponse.split()
        # split to groups by two elements
        groups = zip(*[iter(elements)] * 2)
        # return dict
        return dict(groups)
    
    def __getAuthResponse(self, password, challenge):
        # if I need auth some day :)
    	hpass = hashlib.md5(password).hexdigest()
    	response = hashlib.md5 (challenge + hpass).hexdigest()
    	return response
    
    def __openUrlWithCookie(self, url, cookiejar):
    	cookieHandler = urllib2.HTTPCookieProcessor(cookiejar)
    	opener = urllib2.build_opener(cookieHandler)	    
    	request = urllib2.Request(url)    
    	fp = opener.open(request)
    	text = fp.read()
    	fp.close()    
    	return text
    
    def __buildUserUrl(self, username = ""):
        name = self.m_userName
        if username != "":
            name = username        
        #return "http://" + name + "." + url[LJUserSession.prefixLen:] + "?mode=full"
        return "http://www.livejournal.com/tools/friendlist.bml?user=" + name + "&nopics=1"                    
        #http://www.livejournal.com/tools/friendlist.bml?user=igiss&nopics=1
        
    def __addAndLink(self, parent, child, communityFlag):
        # creates a child node and links parent with the newly created child
        if not self.m_friendsTree.hasNode(child):
            self.m_friendsTree.addNode(child)
            self.m_friendsTree.addNodeData(child, 'C', communityFlag)
            self.m_friendsTree.setRef(parent, child)
        
    def __setNodeDepth(self, nodeName, depth):
        # sets up node depth if it is -1
        # it gaurantees that all the depth will be counted from the root node only
        if self.m_friendsTree.getDepth(nodeName) == None or self.m_friendsTree.getDepth(nodeName) > depth:
            self.m_friendsTree.setDepth(nodeName, depth)
    
    def setThreadsCount(self, val):
        if val <= 0:
            val = 1
        self.m_maxThreads = val
    
    def getUserFriendsHTML(self, stat, username = ""):
        # returns the list of the given LJ user friends        
        urlToOpen = self.__buildUserUrl(username)
        request = urllib2.Request(urlToOpen)
        try:
            fp = urllib2.urlopen(request)
            text = fp.read()
            fp.close()
            return text
        except urllib2.HTTPError:
            # skip user
            LJUserSession.threadLock.acquire()
            stat.m_totalNetErrSkip += 1
            LJUserSession.threadLock.release()
            return None
        except:
            LJUserSession.threadLock.acquire()
            stat.m_totalNetErrSkip += 1
            LJUserSession.threadLock.release()
            return None
            #raise SocketError   
    
    def fetchUserFriendsFromHTML(self, text):
        # parse fetched HTML and extract friends
        # returns friend as an list of tuples
        # each tuple has link to friend's profile as its first element and
        # the friend's name as a second one
        m1 = LJUserSession.reFriendsList.search(text)
        if m1:            
            m2 = LJUserSession.reFetchFriends.findall(m1.group(1))                        
            if m2:
                output = [0] * len(m2)
                idx = 0
                # reduce URL data and add communities flag
                for item in m2:
                    if self.reCheckCommunity.match(item[0]): output[idx] = (True, item[1])
                    else: output[idx] = (False, item[1])
                    idx += 1
                return output
        return None
    
    def fullFetch(self, friendName, stat, curLevel):
        LJUserSession.threadLock.acquire()
        depth = self.m_friendsTree.getNodeData(friendName, 'D')        
        if self.m_friendsTree.hasNode(friendName) and len(self.m_friendsTree.getRefSet(friendName)) != 0 and \
        self.m_friendsTree.getMaxDepth() - 2 != depth:             
            self.__DDD("Using cache for " + friendName)
            tmp = self.m_friendsTree.getRefSet(friendName)
            friendsOfFriend = [0] * len(tmp)
            idx = 0            
            for item in tmp:
                flag = self.m_friendsTree.getNodeData(item, 'C')
                friendsOfFriend[idx] = (flag, item)
                idx += 1
            stat.m_totalCacheUsed += 1
            LJUserSession.threadLock.release()
        else:
            LJUserSession.threadLock.release()
            rawHTML = self.getUserFriendsHTML(stat, friendName)
            if rawHTML == None:                    
                return None
            friendsOfFriend = self.fetchUserFriendsFromHTML(rawHTML)
            
        return friendsOfFriend

    def __traverseFriends(self, stat, friendsList, history, currUser, checkedUsers, curLevel, maxLevel):
        # main recursive method which collects friends hierarchy and saves
        # it as a hashtree data structure
        
        curUserIdx = 0
        totalUsers = len(friendsList)            
        
        if curLevel == maxLevel:
            return
        
        friendsCache = {}
        
        for friendData in friendsList:
                        
            friendName = friendData[1]
            
            communityFlag = False
            if friendData[0]: communityFlag = True
            
            # update progress
            if curLevel == 1:                
                stat.updateProgress((100 * curUserIdx) / totalUsers)                
            
            curUserIdx += 1
            
            # check for cycles
            if friendName in checkedUsers or currUser == friendName:
                # don't scan myself :)                
                self.__addAndLink(currUser, friendName, communityFlag)
                self.__setNodeDepth(friendName, curLevel)
                self.__DDD("Skipping " + friendName)                
                continue
            
            if curLevel == maxLevel - 1:
                # if the next level will be the maximum recursion level
                # don't load the user profile                
                stat.m_totalUsersChecked += 1
                self.__addAndLink(currUser, friendName, communityFlag)
                self.__setNodeDepth(friendName, curLevel)                
                continue
            
            if self.m_skipCommunities:
                # don't scan communities because many of them
                # have too much participants and it causes much more time
                # consumptions to perform a full scan                
                if communityFlag:                    
                    self.__DDD("Skipping community " + friendName)                                    
                    self.__addAndLink(currUser, friendName, communityFlag)
                    self.__setNodeDepth(friendName, curLevel)                  
                    stat.m_commSkipped += 1                    
                    continue                                                
            
            threadsPool = []
            
            if self.m_maxThreads != 1 and not friendsCache.has_key(friendName):
                
                tmp = [(communityFlag, friendName)]
                for f in friendsList:
                    if f[1] != friendName:
                        tmp.append(f)

                # check if we still have free threads
                # and run this user scan in another thread if so            
                for friend in tmp:
                    if not friendsCache.has_key(friend[1]):
                        if len(threadsPool) < self.m_maxThreads:                    
                            thr = MyThread(str(1), "Thread " + str(1), str(1))
                            threadsPool.append(thr)
                            print friend[1]           
                            thr.setParams(self.fullFetch, *(friend[1], friendsCache, stat, curLevel))
                            thr.start()
                                
                for thrIdx in xrange(0, len(threadsPool)):
                    thr = threadsPool[thrIdx]            
                    thr.join()                            
                    
                friendsOfFriend = friendsCache[friendName]
                
            else:
                
                friendsOfFriend = self.fullFetch(friendName, stat, curLevel)
     
            self.__DDD("Checking friend " + friendName)
            stat.m_totalUsersChecked += 1            
            # debug info
            temp = history[:]
            temp.append(friendName)
            self.__DDD("History: " + str(temp))                

            checkedUsers.add(currUser)

            if friendsOfFriend != None:
                #print "state wait lock 3"                
                self.__addAndLink(currUser, friendName, communityFlag)
                self.__setNodeDepth(friendName, curLevel)
                #print "state release lock 3"                
                self.__traverseFriends(stat, friendsOfFriend, temp, friendName, copy.deepcopy(checkedUsers), curLevel + 1, maxLevel)                

    def __DDD(self, val):
        # my own print method, it does nothing
        # but checks if there is a m_verbose flag enabled or not
        if self.m_verbose:
            print val
    
    def setVerbose(self, val):
        self.m_verbose = val
    
    def scanAllFriends(self, maxRecLevel, stat, skipCommunities = True):
        
        startTime = time.time()
        
        rawHTML = self.getUserFriendsHTML(stat)
        friendsData = self.fetchUserFriendsFromHTML(rawHTML)
        
        # prepare initial friends tree
        self.m_friendsTree = HashTree()
        self.m_friendsTree.addNode(self.m_userName)
        self.m_friendsTree.addNodeData(self.m_userName, 'ROOT', 'True')
        self.m_friendsTree.setMaxDepth(maxRecLevel + 2)
                
        self.__setNodeDepth(self.m_userName, 0)
        
        for user in friendsData:
            # add user
            self.m_friendsTree.addNode(user[1])
            self.m_friendsTree.setRef(self.m_userName, user[1])
            # add URL
            self.m_friendsTree.addNodeData(user[1], 'C', user[0])
            # setup depth            
            self.__setNodeDepth(user[1], 1)
                        
        self.m_skipCommunities = skipCommunities    
                
        self.__traverseFriends(stat, friendsData, [self.m_userName], self.m_userName, set([self.m_userName]), 1, maxRecLevel + 1)       
        
        timeTaken = time.time() - startTime

        stat.m_timeTaken = timeTaken        
        return self.m_friendsTree    
    
    def performRPC(self, method, useCookie = True):
        chal = self.__getChalResponse()
        # construct request
        dataDict = {
                    "mode": method,
                    "user": self.m_userName,
                    "includefriendof": 1,
                    #"auth_method": "cookie",
                    #"chal": chal['challenge']
                    "password": self.m_password
                   }
                
        # encode it to send by means of POST
        data = urllib.urlencode(dataDict)
        # create opener
        request = urllib2.Request(self.flatServerUrl, data)
        
        if (useCookie):
            cookieHandler = urllib2.HTTPCookieProcessor()
            opener = urllib2.build_opener(cookieHandler)
            fp = opener.open(request)
        else:
            fp = urllib2.urlopen(request)
                    
        result = fp.read()
        fp.close()
        
        return result

    def loginLJAccount(self):
        # tries to login to livejournal with the given
        # username and password
        response = self.__getChalResponse()
        auth_response = self.__getAuthResponse(self.m_password, response['challenge'])
        
        authData = urllib.urlencode({
        				"chal": response['challenge'],
        				"response": auth_response,
        				"user": self.m_userName
        				})
        
        cookieHandler = urllib2.HTTPCookieProcessor()
        opener = urllib2.build_opener(cookieHandler)                
        request = urllib2.Request(self.authServerUrl, authData)        
        # get cookie
        fp = opener.open(request)
        fp.close()
        
        # check if authorization is ok        
        if len(cookieHandler.cookiejar) < 2:
        	raise AuthError
                
        return cookieHandler.cookiejar

    @staticmethod
    def getStatDataStruct():
        return StatDataStruct()


class StatDataStruct:
    
    def __init__(self):
        self.m_totalUsersChecked = 0
        self.m_totalCacheUsed = 0
        self.m_commSkipped = 0
        self.m_timeTaken = 0
        self.m_totalNetErrSkip = 0
        self.m_lastStr = ""
    
    def updateProgress(self, percent):
        b = '\b' * (len(self.m_lastStr) + 1)
        self.m_lastStr = str(percent) + "%"
        print b + self.m_lastStr,
    
    def showStats(self):                
        print "Users checked: " + str(self.m_totalUsersChecked)
        print "Communities skipped: " + str(self.m_commSkipped)
        print "Cache entries used: " + str(self.m_totalCacheUsed)
        print "Skipped due to network errors: " + str(self.m_totalNetErrSkip)
        print "Time taken: " + str(int(self.m_timeTaken) / 3600) + " hours, " + str(int(self.m_timeTaken) / 60) + " minutes, " + str(int(self.m_timeTaken) % 60) + " seconds"